
from curl_cffi import requests
import re
import os
import shutil
from util.logger import get_looger

logger = get_looger("彼岸图.log", __file__)

logger.info("爬虫启动......")

try:
    root = 'static/彼岸图/'
    if os.path.exists(root):
        shutil.rmtree(root)
    os.makedirs(root)
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    }
    page = 2
    url = f'https://pic.netbian.com/index_{page}.html'

    res = requests.get(url, headers=headers)
    res_html = res.content.decode('gbk')
    # print(res_html)
    items = re.findall(r'<li><a href="/tupian/(.*?).html" target="_blank"><img src="/uploads/allimg/(.*?).jpg" alt="(.*?)" /><b>.*?</b></a></li>',res_html, re.S)
    logger.info(f"找到 缩略图 共{len(items)} 张图片")
    for item in items:

        print(item)
        title = item[2]
        image_url = f'https://pic.netbian.com/uploads/allimg/{item[1]}.jpg'
        headers.update({
            'referer': url
        })
        image_response = requests.get(image_url, headers=headers)
        with open(f'{root}/{title}.jpg', 'wb') as f:
            f.write(image_response.content)
        detail_url = f'https://pic.netbian.com/tupian/{item[0]}.html'
        detail_response = requests.get(detail_url, headers=headers)
        detail_html = detail_response.content.decode('gbk')
        detail_items = re.search(r'<a href="" id="img"><img src="(.*?)" data-pic=".*?</a>', detail_html, re.S).group(1)
        detail_items = f'https://pic.netbian.com{detail_items}'
        logger.info(f"开始爬取{title} 详情图")
        headers.update({
            'referer': detail_url
        })
        preview_response = requests.get(detail_items, headers=headers)
        with open(f"{root}/{title}_preview.jpg", "wb") as f:
            f.write(preview_response.content)


except Exception as e:
    logger.error(e)

